--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Commit dcf722d85f5d1597557653dd1d5e3bc228af0b1b
Parents : 4ffe16b
Author : Mark Qvist <mark@unsigned.io>
Date : 2024-06-03T01:53:54+02:00
Added audio messaging
Changes
7 files changed, 185 insertions(+), 29 deletions(-)
Diff
diff --git a/sbapp/buildozer.spec b/sbapp/buildozer.spec
index 7b4fcd5a..f4d834eb 100644
--- a/sbapp/buildozer.spec
+++ b/sbapp/buildozer.spec
@@ -16,13 +16,13 @@ android.numeric_version = 20240531
# relevant PRs have now been merged in Kivy/P4A, the next release will hopefully allow
# building a non-ancient PyCa/Cryptography distribution again. When this happens, add
# the "cryptography" dependency back in here.
-requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,cryptography
+requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,libogg,libopus,opusfile,numpy,cryptography,pydub,ffpyplayer
android.gradle_dependencies = com.android.support:support-compat:28.0.0
#android.enable_androidx = True
#android.add_aars = patches/support-compat-28.0.0.aar
-p4a.local_recipes = ../Others/python-for-android/pythonforandroid/recipes
+p4a.local_recipes = ../recipes/
icon.filename = %(source.dir)s/assets/icon.png
presplash.filename = %(source.dir)s/assets/presplash_small.png
diff --git a/sbapp/main.py b/sbapp/main.py
index 16a0ca38..7d88867a 100644
--- a/sbapp/main.py
+++ b/sbapp/main.py
@@ -73,12 +73,14 @@ if args.daemon:
NewConv = DaemonElement; Telemetry = DaemonElement; ObjectDetails = DaemonElement; Announces = DaemonElement;
Messages = DaemonElement; ts_format = DaemonElement; messages_screen_kv = DaemonElement; plyer = DaemonElement; multilingual_markup = DaemonElement;
ContentNavigationDrawer = DaemonElement; DrawerList = DaemonElement; IconListItem = DaemonElement; escape_markup = DaemonElement;
+ SoundLoader = DaemonElement;
else:
from kivymd.app import MDApp
app_superclass = MDApp
from kivy.core.window import Window
from kivy.core.clipboard import Clipboard
+ from kivy.core.audio import SoundLoader
from kivy.base import EventLoop
from kivy.clock import Clock
from kivy.lang.builder import Builder
@@ -102,7 +104,7 @@ else:
import kivy.core.image
kivy.core.image.Logger = redirect_log()
- if RNS.vendor.platformutils.get_platform() == "android":
+ if RNS.vendor.platformutils.is_android():
from sideband.core import SidebandCore
import plyer
@@ -228,6 +230,9 @@ class SidebandApp(MDApp):
self.attach_type = None
self.attach_dialog = None
self.rec_dialog = None
+ self.last_msg_audio = None
+ self.msg_sound = None
+ self.audio_msg_mode = LXMF.AM_OPUS_OGG
Window.softinput_mode = "below_target"
self.icon = self.sideband.asset_dir+"/icon.png"
@@ -1238,7 +1243,7 @@ class SidebandApp(MDApp):
self.open_conversations(direction="right")
def message_send_action(self, sender=None):
- if self.messages_view.ids.message_text.text == "":
+ if not (self.attach_type != None and self.attach_path != None) and self.messages_view.ids.message_text.text == "":
return
def cb(dt):
@@ -1265,10 +1270,14 @@ class SidebandApp(MDApp):
else:
msg_content = self.messages_view.ids.message_text.text
+ if msg_content == "":
+ msg_content = " "
+
context_dest = self.messages_view.ids.messages_scrollview.active_conversation
attachment = None
image = None
+ audio = None
if not self.outbound_mode_command and not self.outbound_mode_paper:
if self.attach_type != None and self.attach_path != None:
try:
@@ -1279,6 +1288,11 @@ class SidebandApp(MDApp):
with open(self.attach_path, "rb") as af:
attachment = [fbn, af.read()]
+ if self.attach_type == "audio":
+ if self.audio_msg_mode == LXMF.AM_OPUS_OGG:
+ with open(self.attach_path, "rb") as af:
+ audio = [self.audio_msg_mode, af.read()]
+
elif self.attach_type == "lbimg":
max_size = 320, 320
with PilImage.open(self.attach_path) as im:
@@ -1350,7 +1364,7 @@ class SidebandApp(MDApp):
self.messages_view.ids.messages_scrollview.scroll_y = 0
self.jobs(0)
- elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image):
+ elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image, audio = audio):
self.messages_view.ids.message_text.text = ""
self.messages_view.ids.messages_scrollview.scroll_y = 0
self.jobs(0)
@@ -1496,6 +1510,41 @@ class SidebandApp(MDApp):
ok_button.bind(on_release=ate_dialog.dismiss)
ate_dialog.open()
+ def play_audio_field(self, audio_field):
+ if audio_field[0] == LXMF.AM_OPUS_OGG:
+ audio_type = "ogg"
+ else:
+ return False
+
+ temp_path = self.sideband.rec_cache+"/msg."+audio_type
+
+ if audio_type == "ogg":
+ if self.last_msg_audio != audio_field[1]:
+ self.last_msg_audio = audio_field[1]
+
+ with open(temp_path, "wb") as af:
+ af.write(self.last_msg_audio)
+
+ if not RNS.vendor.platformutils.is_android():
+ self.msg_sound = SoundLoader.load(temp_path)
+
+ if RNS.vendor.platformutils.is_android():
+ if self.msg_sound != None and self.msg_sound._player != None and self.msg_sound._player.isPlaying():
+ self.msg_sound.stop()
+ else:
+ from plyer import audio
+ self.msg_sound = audio
+ self.msg_sound._file_path = temp_path
+ self.msg_sound.play()
+
+ else:
+ if self.msg_sound != None and self.msg_sound.state == "play":
+ self.msg_sound.stop()
+ return True
+ else:
+ self.msg_sound.play()
+ return True
+
def message_record_audio_action(self):
ss = int(dp(18))
if self.rec_dialog == None:
@@ -1506,7 +1555,7 @@ class SidebandApp(MDApp):
from sbapp.plyer import audio
self.msg_audio = audio
- self.msg_audio._file_path = self.sideband.rec_cache+"/msg_rec.aac"
+ self.msg_audio._file_path = self.sideband.rec_cache+"/recording.ogg"
def a_rec_action(sender):
if not self.rec_dialog.recording:
@@ -1555,12 +1604,48 @@ class SidebandApp(MDApp):
a_rec_action(sender)
self.rec_dialog.dismiss()
- self.attach_path = self.msg_audio._file_path
- self.update_message_widgets()
- toast("Attached \""+str(self.attach_path)+"\"")
+ try:
+ if self.audio_msg_mode == LXMF.AM_OPUS_OGG:
+ self.attach_path = self.msg_audio._file_path
+ RNS.log("Using unmodified OPUS data in OGG container", RNS.LOG_DEBUG)
+ else:
+ ap_start = time.time()
+ from pydub import AudioSegment
+ if RNS.vendor.platformutils.is_android():
+ import pyogg
+ else:
+ import sbapp.pyogg as pyogg
- # TODO: Remove
- self.attach_type = "file"
+ opus_file = pyogg.OpusFile(self.msg_audio._file_path)
+
+ audio = AudioSegment(
+ bytes(opus_file.as_array()),
+ frame_rate=opus_file.frequency,
+ sample_width=opus_file.bytes_per_sample,
+ channels=opus_file.channels,
+ )
+ audio = audio.split_to_mono()[0]
+ audio = audio.apply_gain(-audio.max_dBFS)
+
+ if self.audio_msg_mode >= LXMF.AM_CODEC2_450PWB and self.audio_msg_mode <= LXMF.AM_CODEC2_3200:
+ audio = audio.set_frame_rate(8000)
+ audio = audio.set_sample_width(2)
+ samples = audio.get_array_of_samples()
+
+ ap_duration = time.time() - ap_start
+ RNS.log("Audio processing complete in "+RNS.prettytime(ap_duration)+", samples: "+str(len(samples)), RNS.LOG_DEBUG)
+
+ export_path = self.sideband.rec_cache+"/recording.raw"
+ with open(export_path, "wb") as export_file:
+ export_file.write(samples.tobytes())
+ self.attach_path = export_path
+ os.unlink(self.msg_audio._file_path)
+
+ self.update_message_widgets()
+ toast("Added recorded audio to message")
+
+ except Exception as e:
+ RNS.trace_exception(e)
cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18))
rec_item = DialogItem(IconLeftWidget(icon="record"), text="[size="+str(ss)+"]Start Recording[/size]", on_release=a_rec_action)
@@ -1597,7 +1682,7 @@ class SidebandApp(MDApp):
def message_attach_action(self, attach_type=None):
file_attach_types = ["lbimg", "defimg", "hqimg", "file"]
- rec_attach_types = ["lbaudio", "defaudio", "hqaudio"]
+ rec_attach_types = ["audio"]
self.attach_path = None
if attach_type in file_attach_types:
@@ -1621,24 +1706,33 @@ class SidebandApp(MDApp):
def a_file(sender):
self.attach_dialog.dismiss()
self.message_attach_action(attach_type="file")
+ def a_audio_hq(sender):
+ self.attach_dialog.dismiss()
+ self.audio_msg_mode = LXMF.AM_OPUS_OGG
+ self.message_attach_action(attach_type="audio")
def a_audio_lb(sender):
self.attach_dialog.dismiss()
- self.message_attach_action(attach_type="lbaudio")
+ self.audio_msg_mode = LXMF.AM_CODEC2_3200
+ self.message_attach_action(attach_type="audio")
if self.attach_dialog == None:
ss = int(dp(18))
cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18))
+ ad_items = [
+ DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb),
+ DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def),
+ DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq),
+ DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_hq),
+ DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file)]
+
+ if RNS.vendor.platformutils.is_linux():
+ ad_items.pop(3)
+
self.attach_dialog = MDDialog(
title="Add Attachment",
type="simple",
text="Select the type of attachment you want to send with this message\n",
- items=[
- DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb),
- DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def),
- DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq),
- DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_lb),
- DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file),
- ],
+ items=ad_items,
buttons=[ cancel_button ],
width_offset=dp(12),
)
diff --git a/sbapp/plyer/platforms/android/audio.py b/sbapp/plyer/platforms/android/audio.py
index 0b229c45..33739b86 100644
--- a/sbapp/plyer/platforms/android/audio.py
+++ b/sbapp/plyer/platforms/android/audio.py
@@ -29,6 +29,7 @@ class AndroidAudio(Audio):
self._player = None
self._check_thread = None
self._finished_callback = None
+ self._format = "opus"
def _check_playback(self):
while self._player and self._player.isPlaying():
@@ -41,12 +42,24 @@ class AndroidAudio(Audio):
def _start(self):
self._recorder = MediaRecorder()
- self._recorder.setAudioSource(AudioSource.DEFAULT)
- self._recorder.setAudioSamplingRate(44100)
- self._recorder.setAudioEncodingBitRate(128000)
- self._recorder.setAudioChannels(1)
- self._recorder.setOutputFormat(OutputFormat.MPEG_4)
- self._recorder.setAudioEncoder(AudioEncoder.AAC)
+ # AAC Format, decent quality
+ if self._format == "aac":
+ self._recorder.setAudioSource(AudioSource.DEFAULT)
+ self._recorder.setAudioSamplingRate(48000)
+ self._recorder.setAudioEncodingBitRate(128000)
+ self._recorder.setAudioChannels(1)
+ self._recorder.setOutputFormat(OutputFormat.MPEG_4)
+ self._recorder.setAudioEncoder(AudioEncoder.AAC)
+
+ else:
+ # OPUS
+ self._recorder.setAudioSource(AudioSource.DEFAULT)
+ self._recorder.setAudioSamplingRate(48000)
+ self._recorder.setAudioEncodingBitRate(128000)
+ self._recorder.setAudioChannels(1)
+ self._recorder.setOutputFormat(OutputFormat.OGG)
+ self._recorder.setAudioEncoder(AudioEncoder.OPUS)
+
self._recorder.setOutputFile(self.file_path)
self._recorder.prepare()
diff --git a/sbapp/sideband/core.py b/sbapp/sideband/core.py
index f0219834..225149d4 100644
--- a/sbapp/sideband/core.py
+++ b/sbapp/sideband/core.py
@@ -3507,6 +3507,8 @@ class SidebandCore():
fields[LXMF.FIELD_FILE_ATTACHMENTS] = [attachment]
if image != None:
fields[LXMF.FIELD_IMAGE] = image
+ if audio != None:
+ fields[LXMF.FIELD_AUDIO] = audio
lxm = LXMF.LXMessage(dest, source, content, title="", desired_method=desired_method, fields = fields)
diff --git a/sbapp/ui/helpers.py b/sbapp/ui/helpers.py
index a4d3da93..82a95b8d 100644
--- a/sbapp/ui/helpers.py
+++ b/sbapp/ui/helpers.py
@@ -13,6 +13,7 @@ def mdc(color, hue=None):
hue = "400"
return get_color_from_hex(colors[color][hue])
+color_playing = "Amber"
color_received = "LightGreen"
color_delivered = "Blue"
color_paper = "Indigo"
@@ -21,6 +22,8 @@ color_failed = "Red"
color_unknown = "Gray"
intensity_msgs_dark = "800"
intensity_msgs_light = "500"
+intensity_play_dark = "600"
+intensity_play_light = "300"
class ContentNavigationDrawer(Screen):
pass
diff --git a/sbapp/ui/messages.py b/sbapp/ui/messages.py
index a3d8eb44..4decb8e7 100644
--- a/sbapp/ui/messages.py
+++ b/sbapp/ui/messages.py
@@ -34,12 +34,12 @@ if RNS.vendor.platformutils.get_platform() == "android":
import plyer
from sideband.sense import Telemeter, Commands
from ui.helpers import ts_format, file_ts_format, mdc
- from ui.helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light
+ from ui.helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light
else:
import sbapp.plyer as plyer
from sbapp.sideband.sense import Telemeter, Commands
from .helpers import ts_format, file_ts_format, mdc
- from .helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light
+ from .helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light
if RNS.vendor.platformutils.is_darwin():
from PIL import Image as PilImage
@@ -127,8 +127,10 @@ class Messages():
if self.app.sideband.config["dark_ui"]:
intensity_msgs = intensity_msgs_dark
+ intensity_play = intensity_play_dark
else:
intensity_msgs = intensity_msgs_light
+ intensity_play = intensity_play_light
for w in self.widgets:
m = w.m
@@ -161,8 +163,11 @@ class Messages():
if msg["title"]:
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] "+sphrase+prgstr+" "
+ if w.has_audio:
+ w.heading += f"\n[b]Audio Recording Included[/b]"
m["state"] = msg["state"]
+
if msg["state"] == LXMF.LXMessage.DELIVERED:
w.md_bg_color = msg_color = mdc(color_delivered, intensity_msgs)
txstr = time.strftime(ts_format, time.localtime(msg["sent"]))
@@ -170,6 +175,8 @@ class Messages():
if msg["title"]:
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Delivered"
+ if w.has_audio:
+ w.heading += f"\n[b]Audio Recording Included[/b]"
m["state"] = msg["state"]
if msg["method"] == LXMF.LXMessage.PAPER:
@@ -188,6 +195,8 @@ class Messages():
if msg["title"]:
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] On Propagation Net"
+ if w.has_audio:
+ w.heading += f"\n[b]Audio Recording Included[/b]"
m["state"] = msg["state"]
if msg["state"] == LXMF.LXMessage.FAILED:
@@ -198,15 +207,19 @@ class Messages():
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Failed"
m["state"] = msg["state"]
+ if w.has_audio:
+ w.heading += f"\n[b]Audio Recording Included[/b]"
w.dmenu.items.append(w.dmenu.retry_item)
def update_widget(self):
if self.app.sideband.config["dark_ui"]:
intensity_msgs = intensity_msgs_dark
+ intensity_play = intensity_play_dark
mt_color = [1.0, 1.0, 1.0, 0.8]
else:
intensity_msgs = intensity_msgs_light
+ intensity_play = intensity_play_light
mt_color = [1.0, 1.0, 1.0, 0.95]
self.ids.message_text.font_name = self.app.input_font
@@ -230,7 +243,9 @@ class Messages():
extra_telemetry = {}
telemeter = None
image_field = None
+ audio_field = None
has_image = False
+ has_audio = False
attachments_field = None
has_attachment = False
force_markup = False
@@ -277,6 +292,13 @@ class Messages():
except Exception as e:
pass
+ if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_AUDIO in m["lxm"].fields:
+ try:
+ audio_field = m["lxm"].fields[LXMF.FIELD_AUDIO]
+ has_audio = True
+ except Exception as e:
+ pass
+
if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_FILE_ATTACHMENTS in m["lxm"].fields:
if len(m["lxm"].fields[LXMF.FIELD_FILE_ATTACHMENTS]) > 0:
try:
@@ -380,15 +402,36 @@ class Messages():
heading_str += str(attachment[0])+", "
heading_str = heading_str[:-2]
+ if has_audio:
+ heading_str += f"\n[b]Audio Recording Included[/b]"
+
item = ListLXMessageCard(
text=pre_content+message_markup.decode("utf-8")+extra_content,
heading=heading_str,
md_bg_color=msg_color,
)
+ item.lsource = m["source"]
if has_attachment:
item.attachments_field = attachments_field
+ if has_audio:
+ def play_audio(sender):
+ self.app.play_audio_field(sender.audio_field)
+ stored_color = sender.md_bg_color
+ if sender.lsource == self.app.sideband.lxmf_destination.hash:
+ sender.md_bg_color = mdc(color_delivered, intensity_play)
+ else:
+ sender.md_bg_color = mdc(color_received, intensity_play)
+
+ def cb(dt):
+ sender.md_bg_color = stored_color
+ Clock.schedule_once(cb, 0.25)
+
+ item.has_audio = True
+ item.audio_field = audio_field
+ item.bind(on_release=play_audio)
+
if image_field != None:
item.has_image = True
item.image_field = image_field
diff --git a/setup.py b/setup.py
index eacbc77f..87a6fea9 100644
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,8 @@ setuptools.setup(
'sideband=sbapp:main.run',
]
},
- install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "plyer", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7"],
+ # TODO: Include pydub
+ install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7", "pydub", "ffpyplayer"],
extras_require={
"macos": ["pyobjus"],
},
──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────